{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Neural Networks: MNIST image classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "\n",
    "MAX_POOL_SIZE = 5\n",
    "CONVOLUTION_SIZE = 4\n",
    "CONVOLUTION_FILTERS = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (a)\n",
    "\n",
    "Debug function for testing backwards propagation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_grad(fwd, bwd, x, needs_grad_out=True):\n",
    "    \"\"\"\n",
    "    tests if bwd correctly computes the gradient of fwd at x\n",
    "    args:\n",
    "        fwd: forward function\n",
    "        bwd: backward function\n",
    "        x: point at which to evaluate the gradient of fwd\n",
    "        needs_grad_out: how is bwd called? bwd(x, grad_out) or bwd(x)\n",
    "    \"\"\"\n",
    "    cost = lambda y: np.sum(fwd(y)**2)\n",
    "    if needs_grad_out:\n",
    "        grad_out = 2*fwd(x)\n",
    "        grad_x = bwd(x, grad_out)\n",
    "    else:\n",
    "        grad_x = bwd(x)\n",
    "        \n",
    "    h= 1e-6\n",
    "    for i,xi in np.ndenumerate(x):\n",
    "        xp = np.copy(x)\n",
    "        xp[i] = xi+h\n",
    "        xm = np.copy(x)\n",
    "        xm[i] = xi-h\n",
    "        if needs_grad_out:\n",
    "            num_grad = (cost(xp)-cost(xm))/(2*h)\n",
    "        else:\n",
    "            num_grad = (fwd(xp)-fwd(xm))/(2*h)\n",
    "        \n",
    "        error = grad_x[i] - num_grad\n",
    "        if abs(error) > 1e-5:\n",
    "            print(\"test failed\")\n",
    "            print(\"grad_x\", grad_x)\n",
    "            print(\"grad_x[i]\", grad_x[i])\n",
    "            print(\"num_grad\", num_grad)\n",
    "            return\n",
    "    print(\"test passed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**i. backward_softmax**\n",
    "\n",
    "Forward function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_softmax(x):\n",
    "    \"\"\"\n",
    "    Compute softmax function for a single example.\n",
    "    The shape of the input is of size # num classes.\n",
    "\n",
    "    Important Note: You must be careful to avoid overflow for this function. Functions\n",
    "    like softmax have a tendency to overflow when very large numbers like e^10000 are computed.\n",
    "    You will know that your function is overflow resistent when it can handle input like:\n",
    "    np.array([[10000, 10010, 10]]) without issues.\n",
    "\n",
    "    Args:\n",
    "        x: A 1d numpy float array of shape number_of_classes\n",
    "\n",
    "    Returns:\n",
    "        A 1d numpy float array containing the softmax results of shape  number_of_classes\n",
    "    \"\"\"\n",
    "    x = x - np.max(x,axis=0)\n",
    "    exp = np.exp(x)\n",
    "    s = exp / np.sum(exp,axis=0)\n",
    "    return s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For a vector $x$ we have \n",
    "$$ \\text{softmax}(x) = \\frac{\\exp(x)}{\\sum_{k}\\exp(x_k)}. $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A straightforward calculation shows \n",
    "$$ \\frac{\\partial}{\\partial x_i}\\frac{\\exp(x_j)}{\\sum_{k}x_k} = \\begin{cases}\\frac{\\exp(x_j)}{\\sum_{k}\\exp(x_k)}\\left(1-\\frac{\\exp(x_j)}{\\sum_{k}\\exp(x_k)}\\right) &\\text{if } i=j,\\\\\n",
    "-\\frac{\\exp(x_j)}{\\sum_{k}\\exp(x_k)}\\frac{\\exp(x_i)}{\\sum_{k}\\exp(x_k)} &\\text{otherwise.}\\end{cases}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Therefore the Jacobian is given by the (symmetric) matrix\n",
    "$$ \\mathrm d_x \\text{softmax}(x) = \\mathrm{diag}(\\text{softmax}(x) )- \\text{softmax}(x)\\left(\\text{softmax}(x)\\right)^T$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hence by the chainrule\n",
    "$$\\begin{align*} \n",
    "\\nabla_x \\text{loss} &=\\left(\\mathrm{diag}(\\text{softmax}(x) )- \\text{softmax}(x)\\left(\\text{softmax}(x)\\right)^T\\right)\\left(\\nabla_\\text{softmax(x)} \\text{loss}\\right)\\\\  \n",
    "&= \\text{softmax}(x) \\odot \\nabla_\\text{softmax(x)} \\text{loss} - \\text{softmax}(x)\\left(\\text{softmax}(x)^T\\nabla_\\text{softmax(x)}\\text{loss}\\right) ,\n",
    "\\end{align*}$$\n",
    "where $\\odot$ is the Hadamard product (i.e. elementwise multiplication)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_softmax(x, grad_outputs):\n",
    "    \"\"\"\n",
    "    Compute the gradient of the loss with respect to x.\n",
    "\n",
    "    grad_outputs is the gradient of the loss with respect to the outputs of the softmax.\n",
    "\n",
    "    Args:\n",
    "        x: A 1d numpy float array of shape number_of_classes\n",
    "        grad_outputs: A 1d numpy flaot array of shape number_of_classes \n",
    "\n",
    "    Returns:\n",
    "        A 1d numpy float array of the same shape as x with the derivative of the loss with respect to x\n",
    "    \"\"\"\n",
    "    \n",
    "    # *** START CODE HERE ***\n",
    "    s = forward_softmax(x)\n",
    "    return s*grad_outputs  - s * np.dot(s, grad_outputs)\n",
    "    # *** END CODE HERE ***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test passed\n"
     ]
    }
   ],
   "source": [
    "x = 20*np.random.random_sample(size=10)-10\n",
    "test_grad(forward_softmax, backward_softmax,x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**ii. backward_relu**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_relu(x):\n",
    "    \"\"\"\n",
    "    Compute the relu function for the input x.\n",
    "\n",
    "    Args:\n",
    "        x: A numpy float array\n",
    "\n",
    "    Returns:\n",
    "        A numpy float array containing the relu results\n",
    "    \"\"\"\n",
    "\n",
    "    x[x<=0] = 0\n",
    "\n",
    "    return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As \n",
    "$$ \\text{ReLu}(x) = \\max\\{0,x\\}$$\n",
    "we have\n",
    "$$ \\mathrm d_x \\text{ReLu}(x) = \\mathrm{diag}\\left(1\\{x_1>0\\}, \\ldots,1\\{x_n>0\\}\\right) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "and therefore\n",
    "$$ \\nabla_x \\text{loss} = \\left(1\\{x_1>0\\}, \\ldots,1\\{x_n>0\\}\\right)^T \\odot \\nabla_{\\text{ReLu}(x)} \\text{loss}.$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_relu(x, grad_outputs):\n",
    "    \"\"\"\n",
    "    Compute the gradient of the loss with respect to x\n",
    "\n",
    "    Args:\n",
    "        x: A numpy array of arbitrary shape containing the input.\n",
    "        grad_outputs: A numpy array of the same shape of x containing the gradient of the loss with respect\n",
    "            to the output of relu\n",
    "\n",
    "    Returns:\n",
    "        A numpy array of the same shape as x containing the gradients with respect to x.\n",
    "    \"\"\"\n",
    "    # *** START CODE HERE ***\n",
    "    return  (x > 0) * grad_outputs\n",
    "    # *** END CODE HERE ***\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test passed\n"
     ]
    }
   ],
   "source": [
    "x = 20*np.random.sample(size=10) -10\n",
    "test_grad(forward_relu, backward_relu,x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**iii. backward_cross_entropy_loss**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_cross_entropy_loss(probabilities, labels):\n",
    "    \"\"\"\n",
    "    Compute the output from a cross entropy loss layer given the probabilities and labels.\n",
    "\n",
    "    probabilities is of the shape (# classes)\n",
    "    labels is of the shape (# classes)\n",
    "\n",
    "    The output should be a scalar\n",
    "\n",
    "    Returns:\n",
    "        The result of the log loss layer\n",
    "    \"\"\"\n",
    "\n",
    "    result = 0\n",
    "\n",
    "    for i, label in enumerate(labels):\n",
    "        if label == 1:\n",
    "            result += -np.log(probabilities[i])\n",
    "\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From \n",
    "$$CE(prob, label) = -\\sum_{k=1}^K label_k \\log prob_k = - label^T \\log prob$$\n",
    "we get \n",
    "$$ \\nabla_{prob} CE(prob, label) = - label \\odot \\left(\\frac 1{prob_1},\\ldots, \\frac 1{prob_K}\\right)^T.$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_cross_entropy_loss(probabilities, labels):\n",
    "    \"\"\"\n",
    "    Compute the gradient of the cross entropy loss with respect to the probabilities.\n",
    "\n",
    "    probabilities is of the shape (# classes)\n",
    "    labels is of the shape (# classes)\n",
    "\n",
    "    The output should be the gradient with respect to the probabilities.\n",
    "\n",
    "    Returns:\n",
    "        The gradient of the loss with respect to the probabilities.\n",
    "    \"\"\"\n",
    "\n",
    "    # *** START CODE HERE ***\n",
    "    return - labels / probabilities\n",
    "    # *** END CODE HERE ***\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test passed\n"
     ]
    }
   ],
   "source": [
    "labels = np.zeros(10)\n",
    "probs = np.random.sample(10)\n",
    "i = np.random.randint(10)\n",
    "labels[i]=1\n",
    "fwd = lambda x: forward_cross_entropy_loss(x,labels)\n",
    "bwd = lambda x: backward_cross_entropy_loss(x,labels)\n",
    "test_grad(fwd, bwd, probs, needs_grad_out=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**iv. backward_linear**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_linear(weights, bias, data):\n",
    "    \"\"\"\n",
    "    Compute the output from a linear layer with the given weights, bias and data.\n",
    "    weights is of the shape (input # features, output # features)\n",
    "    bias is of the shape (output # features)\n",
    "    data is of the shape (input # features)\n",
    "\n",
    "    The output should be of the shape (output # features)\n",
    "\n",
    "    Returns:\n",
    "        The result of the linear layer\n",
    "    \"\"\"\n",
    "\n",
    "    return data.dot(weights) + bias\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we denote weights, bias, data as $W,b,x$ (where $b,x$ are column vectors), then the forward_linear function has the output $W^Tx + b$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have \n",
    "$$ \\mathrm d_x (W^Tx+b) = W^T$$\n",
    "and \n",
    "$$ \\mathrm d_b (W^Tx+b) = I.$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Therefore \n",
    "$$ \\nabla_x \\text{loss} = W \\nabla_{W^Tx+b} \\text{loss}$$\n",
    "and \n",
    "$$ \\nabla_b \\text{loss} = \\nabla_{W^Tx+b} \\text{loss}.$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For a matrix $V$ of the same shape as $W$ we have\n",
    "$$\\begin{align*}\n",
    "\\mathrm d_W (W^Tx+b)V &= V^Tx\n",
    "\\end{align*}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Therefore\n",
    "$$\\begin{align*}\n",
    "\\mathrm d_W (\\text{loss})V &= \\mathrm d_{W^Tx+b} (\\text{loss})V^Tx\\\\\n",
    "&= \\mathrm{tr}\\left(\\mathrm d_{W^Tx+b} (\\text{loss})V^Tx\\right)\\\\\n",
    "&= \\mathrm{tr}\\left(V^Tx\\mathrm d_{W^Tx+b} (\\text{loss})\\right).\n",
    "\\end{align*}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hence\n",
    "$$\\nabla_W(\\text{loss}) = x\\mathrm d_{W^Tx+b} (\\text{loss}) = x\\left(\\nabla_{W^Tx+b} (\\text{loss})\\right)^T$$\n",
    "(Check my notes to the gradient in problem set 0, if this doesn't make sense to you.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_linear(weights, bias, data, output_grad):\n",
    "    \"\"\"\n",
    "    Compute the gradients of the loss with respect to the parameters of a linear layer.\n",
    "\n",
    "    See forward_linear for information about the shapes of the variables.\n",
    "\n",
    "    output_grad is the gradient of the loss with respect to the output of this layer.\n",
    "\n",
    "    This should return a tuple with three elements:\n",
    "    - The gradient of the loss with respect to the weights\n",
    "    - The gradient of the loss with respect to the bias\n",
    "    - The gradient of the loss with respect to the data\n",
    "    \"\"\"\n",
    "\n",
    "    # *** START CODE HERE ***\n",
    "    grad_data = weights @ output_grad \n",
    "    grad_weights = data[..., np.newaxis] @ output_grad.reshape((1,-1))\n",
    "    grad_bias = output_grad\n",
    "    \n",
    "    return grad_weights, grad_bias, grad_data\n",
    "    # *** END CODE HERE ***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test backward_linear with random inputs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weights:\n",
      "test passed\n",
      "bias:\n",
      "test passed\n",
      "data:\n",
      "test passed\n"
     ]
    }
   ],
   "source": [
    "weights = 4*np.random.sample((50,10))-2\n",
    "bias = 4*np.random.sample(10)-2\n",
    "data = 4*np.random.sample(50)-2\n",
    "fwd_w = lambda w: forward_linear(w, bias, data)\n",
    "bwd_w = lambda w, output_grad: backward_linear(w,bias, data, output_grad)[0]\n",
    "print(\"weights:\")\n",
    "test_grad(fwd_w,bwd_w,weights)\n",
    "\n",
    "fwd_b = lambda b: forward_linear(weights, b, data)\n",
    "bwd_b = lambda b, output_grad: backward_linear(weights,b, data, output_grad)[1]\n",
    "print(\"bias:\")\n",
    "test_grad(fwd_b,bwd_b,bias)\n",
    "\n",
    "fwd_d = lambda d: forward_linear(weights, bias, d)\n",
    "bwd_d = lambda d, output_grad: backward_linear(weights,bias, d, output_grad)[2]\n",
    "print(\"data:\")\n",
    "test_grad(fwd_d,bwd_d,data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**v. backward_convolution**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_convolution(conv_W, conv_b, data):\n",
    "    \"\"\"\n",
    "    Compute the output from a convolutional layer given the weights and data.\n",
    "\n",
    "    conv_W is of the shape (# output channels, # input channels, convolution width, convolution height )\n",
    "    conv_b is of the shape (# output channels)\n",
    "\n",
    "    data is of the shape (# input channels, width, height)\n",
    "\n",
    "    The output should be the result of a convolution and should be of the size:\n",
    "        (# output channels, width - convolution width + 1, height -  convolution height + 1)\n",
    "\n",
    "    Returns:\n",
    "        The output of the convolution as a numpy array\n",
    "    \"\"\"\n",
    "\n",
    "    conv_channels, _, conv_width, conv_height = conv_W.shape\n",
    "\n",
    "    input_channels, input_width, input_height = data.shape\n",
    "\n",
    "    output = np.zeros((conv_channels, input_width - conv_width + 1, input_height - conv_height + 1))\n",
    "\n",
    "    for x in range(input_width - conv_width + 1):\n",
    "        for y in range(input_height - conv_height + 1):\n",
    "            for output_channel in range(conv_channels):\n",
    "                output[output_channel, x, y] = np.sum(\n",
    "                    np.multiply(data[:, x:(x + conv_width), y:(y + conv_height)], conv_W[output_channel, :, :, :])) + conv_b[output_channel]\n",
    "\n",
    "    return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's abbreviate output,out_channel, convolution_bias, convolution_weights, in_channel by $out, oc, c_b, c_W, ic$ and convolution_width, convolution_height, input_width, input_height by $c_w,c_h, i_w,i_h$.\n",
    "\n",
    "Then\n",
    "$$ out[oc,x,y] = c_b[oc] + \\sum_{di,dj,ic} data[ic, x+di,y+dj] \\cdot c_W[oc, ic, di,dj].$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So\n",
    "$$\\begin{align*} \n",
    "\\frac{\\partial out[oc,x,y]}{\\partial c_W[oc',ic,di,dj]} &= \\begin{cases}\n",
    "0 & \\text{if } oc\\not= oc',\\\\\n",
    "data[ic, x+di,y+dj] &\\text{if } oc = oc',\n",
    "\\end{cases}\\\\\n",
    "\\end{align*}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\frac{\\partial out[oc,x,y]}{\\partial c_b[oc']} = 1\\{oc=oc'\\}, $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "and\n",
    "$$ \\frac{\\partial out[oc,x,y]}{\\partial data[ic,x',y']} = \\begin{cases}\n",
    "c_W[oc,ic,x'-x,y'-y] &\\text{if } 0\\leq x'-x< c_w \\land 0\\leq y'-y< c_h,\\\\\n",
    "0 &\\text{otherwise}.\n",
    "\\end{cases}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Therefore \n",
    "$$\\frac{\\partial \\text{loss}}{\\partial c_W[oc,ic,di,dj]} = \\sum_{x,y}\\frac{\\partial \\text{loss}}{\\partial out[oc,x,y]}data[ic,x+d_i,y+d_j],$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\frac{\\partial \\text{loss}}{\\partial c_b[oc]} = \\sum_{x,y}\\frac{\\partial \\text{loss}}{\\partial out[oc,x,y]},$$\n",
    "where in both cases $0\\leq x \\leq i_w-c_w$ and $0\\leq y \\leq i_h-c_h$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lastly\n",
    "$$\\frac{\\partial \\text{loss}}{\\partial data[ic,x',y']} = \\sum_{oc,x,y} \\frac{\\partial \\text{loss}}{\\partial out[oc,x,y]}c_W[oc,ic,x'-x,y'-y]$$\n",
    "where again $0\\leq x \\leq i_w-c_w$ and $0\\leq y \\leq i_h-c_h$, but also $x'-c_w+1\\leq x\\leq x'$ and $y'-c_h+1\\leq y\\leq y'$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_convolution(conv_W, conv_b, data, output_grad):\n",
    "    \"\"\"\n",
    "    Compute the gradient of the loss with respect to the parameters of the convolution.\n",
    "\n",
    "    See forward_convolution for the sizes of the arguments.\n",
    "    output_grad is the gradient of the loss with respect to the output of the convolution.\n",
    "\n",
    "    Returns:\n",
    "        A tuple containing 3 gradients.\n",
    "        The first element is the gradient of the loss with respect to the convolution weights\n",
    "        The second element is the gradient of the loss with respect to the convolution bias\n",
    "        The third element is the gradient of the loss with respect to the input data\n",
    "    \"\"\"\n",
    "\n",
    "    # *** START CODE HERE ***\n",
    "    conv_W_grad = np.zeros(conv_W.shape)\n",
    "    data_grad = np.zeros(data.shape)\n",
    "    \n",
    "    conv_channels, _, conv_width, conv_height = conv_W.shape\n",
    "\n",
    "    input_channels, input_width, input_height = data.shape\n",
    "    \n",
    "    conv_b_grad = np.sum(output_grad, axis=(1,2))\n",
    "    \n",
    "    for di in range(conv_width):\n",
    "        for dj in range(conv_height):\n",
    "            conv_W_grad[:,:,di,dj] = np.sum(output_grad.reshape((conv_channels,1,input_width-conv_width+1,input_height-conv_height+1)) * data[:,di:di+input_width-conv_width+1,dj:dj+input_height-conv_height+1], axis=(-2,-1))\n",
    "    \n",
    "    for x_ in range(input_width):\n",
    "        xmin = np.max([0, x_-conv_width+1])\n",
    "        xmax = np.min([input_width-conv_width, x_])\n",
    "        for y_ in range(input_height):\n",
    "            ymin = np.max([0, y_-conv_height+1])\n",
    "            ymax = np.min([input_height-conv_height, y_])            \n",
    "            \n",
    "            for x in range(xmin, xmax+1):\n",
    "                for y in range(ymin, ymax+1):\n",
    "                    data_grad[:,x_,y_]+=np.sum(output_grad[:,x,y][...,np.newaxis]*conv_W[:,:,x_-x,y_-y], axis=0)\n",
    "    \n",
    "    return conv_W_grad, conv_b_grad, data_grad            \n",
    "    # *** END CODE HERE ***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test it (the data test takes a bit):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weights:\n",
      "test passed\n",
      "bias:\n",
      "test passed\n",
      "data:\n",
      "test passed\n"
     ]
    }
   ],
   "source": [
    "conv_W = 4*np.random.sample((2,1,4,4))-2\n",
    "conv_b = 4*np.random.sample(2)-2\n",
    "data = 4*np.random.sample((1,28,28))-2\n",
    "\n",
    "fwd_w = lambda w: forward_convolution(w, conv_b, data)\n",
    "bwd_w = lambda w, output_grad: backward_convolution(w,conv_b, data, output_grad)[0]\n",
    "print(\"weights:\")\n",
    "test_grad(fwd_w,bwd_w,conv_W)\n",
    "\n",
    "fwd_b = lambda b: forward_convolution(conv_W, b, data)\n",
    "bwd_b = lambda b, output_grad: backward_convolution(conv_W,b, data, output_grad)[1]\n",
    "print(\"bias:\")\n",
    "test_grad(fwd_b,bwd_b,conv_b)\n",
    "\n",
    "fwd_d = lambda d: forward_convolution(conv_W, conv_b, d)\n",
    "bwd_d = lambda d, output_grad: backward_convolution(conv_W,conv_b, d, output_grad)[2]\n",
    "print(\"data:\")\n",
    "test_grad(fwd_d,bwd_d,data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**vi. backward_max_pool**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_max_pool(data, pool_width, pool_height):\n",
    "    \"\"\"\n",
    "    Compute the output from a max pooling layer given the data and pool dimensions.\n",
    "\n",
    "    The stride length should be equal to the pool size\n",
    "\n",
    "    data is of the shape (# channels, width, height)\n",
    "\n",
    "    The output should be the result of the max pooling layer and should be of size:\n",
    "        (# channels, width // pool_width, height // pool_height)\n",
    "\n",
    "    Returns:\n",
    "        The result of the max pooling layer\n",
    "    \"\"\"\n",
    "\n",
    "    input_channels, input_width, input_height = data.shape\n",
    "\n",
    "    output = np.zeros((input_channels, input_width // pool_width, input_height // pool_height))\n",
    "\n",
    "    for x in range(0, input_width, pool_width):\n",
    "        for y in range(0, input_height, pool_height):\n",
    "\n",
    "            output[:, x // pool_width, y // pool_height] = np.amax(data[:, x:(x + pool_width), y:(y + pool_height)], axis=(1, 2))\n",
    "\n",
    "    return output\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With \n",
    "$$out[c,x,y] = \\max_{di,dj} data[c, x\\cdot p_w+di, y\\cdot p_h+dj],$$\n",
    "where $p_w,p_h$ are the pool width and height, we get"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ \\frac{\\partial out[c,x,y]}{\\partial data[c',x',y']}= 1$$\n",
    "if $c=c'$ and $(x',y') = (xp_w+di^*, yp_h+dj^*)$ for \n",
    "$$ (di^*,dj^*) = \\arg\\max_{di,dj} data[c, x\\cdot p_w+di, y\\cdot p_h+dj].$$\n",
    "Otherwise \n",
    "$$ \\frac{\\partial out[c,x,y]}{\\partial data[c',x',y']}= 0$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Therefore \n",
    "$$ \\frac{\\partial \\text{loss}}{\\partial data[c,x',y']}= \\sum_{x,y} \\frac{\\partial \\text{loss}}{\\partial out[c,x,y]}$$\n",
    "where the sum ranges over all $x,y$ such that $\\frac{\\partial out[c,x,y]}{\\partial data[c,x',y']}= 1$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_max_pool(data, pool_width, pool_height, output_grad):\n",
    "    \"\"\"\n",
    "    Compute the gradient of the loss with respect to the data in the max pooling layer.\n",
    "\n",
    "    data is of the shape (# channels, width, height)\n",
    "    output_grad is of shape (# channels, width // pool_width, height // pool_height)\n",
    "\n",
    "    output_grad is the gradient of the loss with respect to the output of the backward max\n",
    "    pool layer.\n",
    "\n",
    "    Returns:\n",
    "        The gradient of the loss with respect to the data (of same shape as data)\n",
    "    \"\"\"\n",
    "\n",
    "    # *** START CODE HERE ***\n",
    "    input_channels, input_width, input_height = data.shape\n",
    "    output_width, output_height = output_grad.shape[1:]\n",
    "    forw_data = forward_max_pool(data, pool_width, pool_height)\n",
    "    \n",
    "    data_grad = np.zeros(data.shape)\n",
    "    for x in range(0, input_width, pool_width):\n",
    "        for y in range(0, input_height, pool_height):\n",
    "            for c in range(0,input_channels):\n",
    "                pool = data[c, x:(x + pool_width), y:(y + pool_height)]\n",
    "                argmax = np.argmax(pool)\n",
    "                di,dj = np.unravel_index(argmax, pool.shape)\n",
    "                data_grad[c,x+di,y+dj] += output_grad[c, x//pool_width, y//pool_height]\n",
    "    \n",
    "    return data_grad\n",
    "    # *** END CODE HERE ***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data:\n",
      "test passed\n"
     ]
    }
   ],
   "source": [
    "data = 10*np.random.sample((2,25,25))-5\n",
    "pool_width = 5\n",
    "pool_height = 5\n",
    "\n",
    "\n",
    "fwd = lambda d: forward_max_pool(d, pool_width, pool_height)\n",
    "bwd = lambda d, output_grad: backward_max_pool(d, pool_width, pool_height,output_grad)\n",
    "print(\"data:\")\n",
    "test_grad(fwd,bwd,data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (b) backward_prop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_prop(data, labels, params):\n",
    "    \"\"\"\n",
    "    Implement the forward layer given the data, labels, and params.\n",
    "    \n",
    "    Args:\n",
    "        data: A numpy array containing the input (shape is 1 by 28 by 28)\n",
    "        labels: A 1d numpy array containing the labels (shape is 10)\n",
    "        params: A dictionary mapping parameter names to numpy arrays with the parameters.\n",
    "            This numpy array will contain W1, b1, W2 and b2\n",
    "            W1 and b1 represent the weights and bias for the hidden layer of the network\n",
    "            W2 and b2 represent the weights and bias for the output layer of the network\n",
    "\n",
    "    Returns:\n",
    "        A 2 element tuple containing:\n",
    "            1. A numpy array The output (after the softmax) of the output layer\n",
    "            2. The average loss for these data elements\n",
    "    \"\"\"\n",
    "\n",
    "    W1 = params['W1']\n",
    "    b1 = params['b1']\n",
    "    W2 = params['W2']\n",
    "    b2 = params['b2']\n",
    "\n",
    "    first_convolution = forward_convolution(W1, b1, data)\n",
    "    first_max_pool = forward_max_pool(first_convolution, MAX_POOL_SIZE, MAX_POOL_SIZE)\n",
    "    first_after_relu = forward_relu(first_max_pool)\n",
    "\n",
    "    flattened = np.reshape(first_after_relu, (-1))\n",
    "   \n",
    "    logits = forward_linear(W2, b2, flattened)\n",
    "\n",
    "    y = forward_softmax(logits)\n",
    "    cost = forward_cross_entropy_loss(y, labels)\n",
    "\n",
    "    return y, cost\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_prop(data, labels, params):\n",
    "    \"\"\"\n",
    "    Implement the backward propegation gradient computation step for a neural network\n",
    "    \n",
    "    Args:\n",
    "        data: A numpy array containing the input for a single example\n",
    "        labels: A 1d numpy array containing the labels for a single example\n",
    "        params: A dictionary mapping parameter names to numpy arrays with the parameters.\n",
    "            This numpy array will contain W1, b1, W2, and b2\n",
    "            W1 and b1 represent the weights and bias for the convolutional layer\n",
    "            W2 and b2 represent the weights and bias for the output layer of the network\n",
    "\n",
    "    Returns:\n",
    "        A dictionary of strings to numpy arrays where each key represents the name of a weight\n",
    "        and the values represent the gradient of the loss with respect to that weight.\n",
    "        \n",
    "        In particular, it should have 4 elements:\n",
    "            W1, W2, b1, and b2\n",
    "    \"\"\"\n",
    "    # *** START CODE HERE ***\n",
    "    W1 = params['W1']\n",
    "    b1 = params['b1']\n",
    "    W2 = params['W2']\n",
    "    b2 = params['b2']\n",
    "    \n",
    "    first_convolution = forward_convolution(W1, b1, data)\n",
    "    first_max_pool = forward_max_pool(first_convolution, MAX_POOL_SIZE, MAX_POOL_SIZE)\n",
    "    first_after_relu = forward_relu(first_max_pool)\n",
    "\n",
    "    flattened = np.reshape(first_after_relu, (-1))\n",
    "   \n",
    "    logits = forward_linear(W2, b2, flattened)\n",
    "\n",
    "    y = forward_softmax(logits)\n",
    "    cost = forward_cross_entropy_loss(y, labels)\n",
    "    \n",
    "    grad={}\n",
    "    grad_y = backward_cross_entropy_loss(y, labels)\n",
    "    grad_logits =backward_softmax(logits, grad_y)\n",
    "    grad['W2'], grad['b2'], grad_flattened = backward_linear(W2, b2, flattened, grad_logits)\n",
    "\n",
    "    grad_first_after_relu = grad_flattened.reshape(first_after_relu.shape)\n",
    "    grad_first_max_pool = backward_relu(first_max_pool, grad_first_after_relu)\n",
    "    grad_first_convolution = backward_max_pool(first_convolution, MAX_POOL_SIZE, MAX_POOL_SIZE, grad_first_max_pool)\n",
    "    grad['W1'], grad['b1'], grad_data = backward_convolution(W1, b1, data, grad_first_convolution)\n",
    "    return grad\n",
    "    # *** END CODE HERE ***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the model\n",
    "\n",
    "I left the following functions from the starter code unchanged: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_initial_params():\n",
    "    \"\"\"\n",
    "    Compute the initial parameters for the neural network.\n",
    "\n",
    "    This function should return a dictionary mapping parameter names to numpy arrays containing\n",
    "    the initial values for those parameters.\n",
    "\n",
    "    There should be four parameters for this model:\n",
    "    W1 is the weight matrix for the convolutional layer\n",
    "    b1 is the bias vector for the convolutional layer\n",
    "    W2 is the weight matrix for the output layers\n",
    "    b2 is the bias vector for the output layer\n",
    "\n",
    "    Weight matrices should be initialized with values drawn from a random normal distribution.\n",
    "    The mean of that distribution should be 0.\n",
    "    The variance of that distribution should be 1/sqrt(n) where n is the number of neurons that \n",
    "    feed into an output for that layer.\n",
    "\n",
    "    Bias vectors should be initialized with zero.\n",
    "    \n",
    "    \n",
    "    Returns:\n",
    "        A dict mapping parameter names to numpy arrays\n",
    "    \"\"\"\n",
    "  \n",
    "    size_after_convolution = 28 - CONVOLUTION_SIZE + 1\n",
    "    size_after_max_pooling = size_after_convolution // MAX_POOL_SIZE\n",
    "\n",
    "    num_hidden = size_after_max_pooling * size_after_max_pooling * CONVOLUTION_FILTERS\n",
    "\n",
    "    return {\n",
    "        'W1': np.random.normal(size = (CONVOLUTION_FILTERS, 1, CONVOLUTION_SIZE, CONVOLUTION_SIZE), scale=1/ math.sqrt(CONVOLUTION_SIZE * CONVOLUTION_SIZE)),\n",
    "        'b1': np.zeros(CONVOLUTION_FILTERS),\n",
    "        'W2': np.random.normal(size = (num_hidden, 10), scale = 1/ math.sqrt(num_hidden)),\n",
    "        'b2': np.zeros(10)\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_prop_batch(batch_data, batch_labels, params, forward_prop_func):\n",
    "    \"\"\"Apply the forward prop func to every image in a batch\"\"\"\n",
    "\n",
    "    y_array = []\n",
    "    cost_array = []\n",
    "\n",
    "    for item, label in zip(batch_data, batch_labels):\n",
    "        y, cost = forward_prop_func(item, label, params)\n",
    "        y_array.append(y)\n",
    "        cost_array.append(cost)\n",
    "\n",
    "    return np.array(y_array), np.array(cost_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent_batch(batch_data, batch_labels, learning_rate, params, backward_prop_func):\n",
    "    \"\"\"\n",
    "    Perform one batch of gradient descent on the given training data using the provided learning rate.\n",
    "\n",
    "    This code should update the parameters stored in params.\n",
    "    It should not return anything\n",
    "\n",
    "    Args:\n",
    "        batch_data: A numpy array containing the training data for the batch\n",
    "        train_labels: A numpy array containing the training labels for the batch\n",
    "        learning_rate: The learning rate\n",
    "        params: A dict of parameter names to parameter values that should be updated.\n",
    "        backward_prop_func: A function that follows the backwards_prop API\n",
    "\n",
    "    Returns: This function returns nothing.\n",
    "    \"\"\"\n",
    "\n",
    "    total_grad = {}\n",
    "\n",
    "    for i in range(batch_data.shape[0]):\n",
    "        grad = backward_prop_func(\n",
    "            batch_data[i, :, :], \n",
    "            batch_labels[i, :], \n",
    "            params)\n",
    "        for key, value in grad.items():\n",
    "            if key not in total_grad:\n",
    "                total_grad[key] = np.zeros(value.shape)\n",
    "\n",
    "            total_grad[key] += value\n",
    "\n",
    "    params['W1'] = params['W1'] - learning_rate * total_grad['W1']\n",
    "    params['W2'] = params['W2'] - learning_rate * total_grad['W2']\n",
    "    params['b1'] = params['b1'] - learning_rate * total_grad['b1']\n",
    "    params['b2'] = params['b2'] - learning_rate * total_grad['b2']\n",
    "\n",
    "    # This function does not return anything\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nn_test(data, labels, params):\n",
    "    output, cost = forward_pr(data, labels, params)\n",
    "    accuracy = compute_accuracy(output, labels)\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_accuracy(output, labels):\n",
    "    correct_output = np.argmax(output,axis=1)\n",
    "    correct_labels = np.argmax(labels,axis=1)\n",
    "\n",
    "    is_correct = [a == b for a,b in zip(correct_output, correct_labels)]\n",
    "\n",
    "    accuracy = sum(is_correct) * 1. / labels.shape[0]\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_labels(labels):\n",
    "    one_hot_labels = np.zeros((labels.size, 10))\n",
    "    one_hot_labels[np.arange(labels.size),labels.astype(int)] = 1\n",
    "    return one_hot_labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data(images_file, labels_file):\n",
    "    x = np.loadtxt(images_file, delimiter=',')\n",
    "    y = np.loadtxt(labels_file, delimiter=',')\n",
    "\n",
    "    x = np.reshape(x, (x.shape[0], 1, 28, 28))\n",
    "\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I made some minor changes here to plot also the final cost and accuracy after 400 batches:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nn_train(\n",
    "    train_data, train_labels, dev_data, dev_labels, \n",
    "    get_initial_params_func, forward_prop_func, backward_prop_func,\n",
    "    learning_rate=5, batch_size=16, num_batches=400):\n",
    "\n",
    "    m = train_data.shape[0]\n",
    "\n",
    "    params = get_initial_params_func()\n",
    "\n",
    "    cost_dev = []\n",
    "    accuracy_dev = []\n",
    "    \n",
    "    \n",
    "    for batch in range(num_batches):\n",
    "        print('Currently processing {} / {}'.format(batch, num_batches))\n",
    "\n",
    "        batch_data = train_data[batch * batch_size:(batch + 1) * batch_size, :, :, :]\n",
    "        batch_labels = train_labels[batch * batch_size: (batch + 1) * batch_size, :]\n",
    "\n",
    "        if batch % 100 == 0 or batch == num_batches-1:\n",
    "            output, cost = forward_prop_batch(dev_data, dev_labels, params, forward_prop_func)\n",
    "            cost_dev.append(sum(cost) / len(cost))\n",
    "            accuracy_dev.append(compute_accuracy(output, dev_labels))\n",
    "            \n",
    "            print('Cost and accuracy (dev)', cost_dev[-1], accuracy_dev[-1])\n",
    "\n",
    "        gradient_descent_batch(batch_data, batch_labels, \n",
    "            learning_rate, params, backward_prop_func)\n",
    "\n",
    "    return params, cost_dev, accuracy_dev\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I changed it to not save the plot:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_train(all_data, all_labels, backward_prop_func):\n",
    "    params, cost_dev, accuracy_dev = nn_train(\n",
    "        all_data['train'], all_labels['train'], \n",
    "        all_data['dev'], all_labels['dev'],\n",
    "        get_initial_params, forward_prop, backward_prop_func,\n",
    "        learning_rate=1e-2, batch_size=16, num_batches=400\n",
    "    )\n",
    "\n",
    "    t = np.arange(400 // 100 +1)\n",
    "\n",
    "    fig, (ax1, ax2) = plt.subplots(2, 1)\n",
    "\n",
    "    ax1.plot(t, cost_dev, label='dev loss')\n",
    "    ax1.legend()\n",
    "    ax1.set_xlabel('time')\n",
    "    ax1.set_ylabel('loss')\n",
    "    ax1.set_title('Training curve')\n",
    "\n",
    "    ax2.plot(t, accuracy_dev, label='dev acc')\n",
    "    ax2.legend()\n",
    "    ax2.set_xlabel('time')\n",
    "    ax2.set_ylabel('accuracy')\n",
    "    \n",
    "#     fig.savefig('output/train.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can load and prepare the data (this takes about half a minute):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#def main():\n",
    "    np.random.seed(100)\n",
    "    train_data, train_labels = read_data('data/images_train.csv', 'data/labels_train.csv')\n",
    "    train_labels = one_hot_labels(train_labels)\n",
    "    p = np.random.permutation(60000)\n",
    "    train_data = train_data[p,:]\n",
    "    train_labels = train_labels[p,:]\n",
    "\n",
    "    dev_data = train_data[0:400,:]\n",
    "    dev_labels = train_labels[0:400,:]\n",
    "    train_data = train_data[400:,:]\n",
    "    train_labels = train_labels[400:,:]\n",
    "\n",
    "    mean = np.mean(train_data)\n",
    "    std = np.std(train_data)\n",
    "    train_data = (train_data - mean) / std\n",
    "    dev_data = (dev_data - mean) / std\n",
    "\n",
    "    all_data = {\n",
    "        'train': train_data,\n",
    "        'dev': dev_data,\n",
    "    }\n",
    "\n",
    "    all_labels = {\n",
    "        'train': train_labels,\n",
    "        'dev': dev_labels,\n",
    "    }\n",
    "    \n",
    "    #run_train(all_data, all_labels, backward_prop)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally we can train the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Currently processing 0 / 400\n",
      "Cost and accuracy (dev) 2.721417647426753 0.0725\n",
      "Currently processing 1 / 400\n",
      "Currently processing 2 / 400\n",
      "Currently processing 3 / 400\n",
      "Currently processing 4 / 400\n",
      "Currently processing 5 / 400\n",
      "Currently processing 6 / 400\n",
      "Currently processing 7 / 400\n",
      "Currently processing 8 / 400\n",
      "Currently processing 9 / 400\n",
      "Currently processing 10 / 400\n",
      "Currently processing 11 / 400\n",
      "Currently processing 12 / 400\n",
      "Currently processing 13 / 400\n",
      "Currently processing 14 / 400\n",
      "Currently processing 15 / 400\n",
      "Currently processing 16 / 400\n",
      "Currently processing 17 / 400\n",
      "Currently processing 18 / 400\n",
      "Currently processing 19 / 400\n",
      "Currently processing 20 / 400\n",
      "Currently processing 21 / 400\n",
      "Currently processing 22 / 400\n",
      "Currently processing 23 / 400\n",
      "Currently processing 24 / 400\n",
      "Currently processing 25 / 400\n",
      "Currently processing 26 / 400\n",
      "Currently processing 27 / 400\n",
      "Currently processing 28 / 400\n",
      "Currently processing 29 / 400\n",
      "Currently processing 30 / 400\n",
      "Currently processing 31 / 400\n",
      "Currently processing 32 / 400\n",
      "Currently processing 33 / 400\n",
      "Currently processing 34 / 400\n",
      "Currently processing 35 / 400\n",
      "Currently processing 36 / 400\n",
      "Currently processing 37 / 400\n",
      "Currently processing 38 / 400\n",
      "Currently processing 39 / 400\n",
      "Currently processing 40 / 400\n",
      "Currently processing 41 / 400\n",
      "Currently processing 42 / 400\n",
      "Currently processing 43 / 400\n",
      "Currently processing 44 / 400\n",
      "Currently processing 45 / 400\n",
      "Currently processing 46 / 400\n",
      "Currently processing 47 / 400\n",
      "Currently processing 48 / 400\n",
      "Currently processing 49 / 400\n",
      "Currently processing 50 / 400\n",
      "Currently processing 51 / 400\n",
      "Currently processing 52 / 400\n",
      "Currently processing 53 / 400\n",
      "Currently processing 54 / 400\n",
      "Currently processing 55 / 400\n",
      "Currently processing 56 / 400\n",
      "Currently processing 57 / 400\n",
      "Currently processing 58 / 400\n",
      "Currently processing 59 / 400\n",
      "Currently processing 60 / 400\n",
      "Currently processing 61 / 400\n",
      "Currently processing 62 / 400\n",
      "Currently processing 63 / 400\n",
      "Currently processing 64 / 400\n",
      "Currently processing 65 / 400\n",
      "Currently processing 66 / 400\n",
      "Currently processing 67 / 400\n",
      "Currently processing 68 / 400\n",
      "Currently processing 69 / 400\n",
      "Currently processing 70 / 400\n",
      "Currently processing 71 / 400\n",
      "Currently processing 72 / 400\n",
      "Currently processing 73 / 400\n",
      "Currently processing 74 / 400\n",
      "Currently processing 75 / 400\n",
      "Currently processing 76 / 400\n",
      "Currently processing 77 / 400\n",
      "Currently processing 78 / 400\n",
      "Currently processing 79 / 400\n",
      "Currently processing 80 / 400\n",
      "Currently processing 81 / 400\n",
      "Currently processing 82 / 400\n",
      "Currently processing 83 / 400\n",
      "Currently processing 84 / 400\n",
      "Currently processing 85 / 400\n",
      "Currently processing 86 / 400\n",
      "Currently processing 87 / 400\n",
      "Currently processing 88 / 400\n",
      "Currently processing 89 / 400\n",
      "Currently processing 90 / 400\n",
      "Currently processing 91 / 400\n",
      "Currently processing 92 / 400\n",
      "Currently processing 93 / 400\n",
      "Currently processing 94 / 400\n",
      "Currently processing 95 / 400\n",
      "Currently processing 96 / 400\n",
      "Currently processing 97 / 400\n",
      "Currently processing 98 / 400\n",
      "Currently processing 99 / 400\n",
      "Currently processing 100 / 400\n",
      "Cost and accuracy (dev) 0.6413721697623075 0.78\n",
      "Currently processing 101 / 400\n",
      "Currently processing 102 / 400\n",
      "Currently processing 103 / 400\n",
      "Currently processing 104 / 400\n",
      "Currently processing 105 / 400\n",
      "Currently processing 106 / 400\n",
      "Currently processing 107 / 400\n",
      "Currently processing 108 / 400\n",
      "Currently processing 109 / 400\n",
      "Currently processing 110 / 400\n",
      "Currently processing 111 / 400\n",
      "Currently processing 112 / 400\n",
      "Currently processing 113 / 400\n",
      "Currently processing 114 / 400\n",
      "Currently processing 115 / 400\n",
      "Currently processing 116 / 400\n",
      "Currently processing 117 / 400\n",
      "Currently processing 118 / 400\n",
      "Currently processing 119 / 400\n",
      "Currently processing 120 / 400\n",
      "Currently processing 121 / 400\n",
      "Currently processing 122 / 400\n",
      "Currently processing 123 / 400\n",
      "Currently processing 124 / 400\n",
      "Currently processing 125 / 400\n",
      "Currently processing 126 / 400\n",
      "Currently processing 127 / 400\n",
      "Currently processing 128 / 400\n",
      "Currently processing 129 / 400\n",
      "Currently processing 130 / 400\n",
      "Currently processing 131 / 400\n",
      "Currently processing 132 / 400\n",
      "Currently processing 133 / 400\n",
      "Currently processing 134 / 400\n",
      "Currently processing 135 / 400\n",
      "Currently processing 136 / 400\n",
      "Currently processing 137 / 400\n",
      "Currently processing 138 / 400\n",
      "Currently processing 139 / 400\n",
      "Currently processing 140 / 400\n",
      "Currently processing 141 / 400\n",
      "Currently processing 142 / 400\n",
      "Currently processing 143 / 400\n",
      "Currently processing 144 / 400\n",
      "Currently processing 145 / 400\n",
      "Currently processing 146 / 400\n",
      "Currently processing 147 / 400\n",
      "Currently processing 148 / 400\n",
      "Currently processing 149 / 400\n",
      "Currently processing 150 / 400\n",
      "Currently processing 151 / 400\n",
      "Currently processing 152 / 400\n",
      "Currently processing 153 / 400\n",
      "Currently processing 154 / 400\n",
      "Currently processing 155 / 400\n",
      "Currently processing 156 / 400\n",
      "Currently processing 157 / 400\n",
      "Currently processing 158 / 400\n",
      "Currently processing 159 / 400\n",
      "Currently processing 160 / 400\n",
      "Currently processing 161 / 400\n",
      "Currently processing 162 / 400\n",
      "Currently processing 163 / 400\n",
      "Currently processing 164 / 400\n",
      "Currently processing 165 / 400\n",
      "Currently processing 166 / 400\n",
      "Currently processing 167 / 400\n",
      "Currently processing 168 / 400\n",
      "Currently processing 169 / 400\n",
      "Currently processing 170 / 400\n",
      "Currently processing 171 / 400\n",
      "Currently processing 172 / 400\n",
      "Currently processing 173 / 400\n",
      "Currently processing 174 / 400\n",
      "Currently processing 175 / 400\n",
      "Currently processing 176 / 400\n",
      "Currently processing 177 / 400\n",
      "Currently processing 178 / 400\n",
      "Currently processing 179 / 400\n",
      "Currently processing 180 / 400\n",
      "Currently processing 181 / 400\n",
      "Currently processing 182 / 400\n",
      "Currently processing 183 / 400\n",
      "Currently processing 184 / 400\n",
      "Currently processing 185 / 400\n",
      "Currently processing 186 / 400\n",
      "Currently processing 187 / 400\n",
      "Currently processing 188 / 400\n",
      "Currently processing 189 / 400\n",
      "Currently processing 190 / 400\n",
      "Currently processing 191 / 400\n",
      "Currently processing 192 / 400\n",
      "Currently processing 193 / 400\n",
      "Currently processing 194 / 400\n",
      "Currently processing 195 / 400\n",
      "Currently processing 196 / 400\n",
      "Currently processing 197 / 400\n",
      "Currently processing 198 / 400\n",
      "Currently processing 199 / 400\n",
      "Currently processing 200 / 400\n",
      "Cost and accuracy (dev) 0.4348282221098257 0.8625\n",
      "Currently processing 201 / 400\n",
      "Currently processing 202 / 400\n",
      "Currently processing 203 / 400\n",
      "Currently processing 204 / 400\n",
      "Currently processing 205 / 400\n",
      "Currently processing 206 / 400\n",
      "Currently processing 207 / 400\n",
      "Currently processing 208 / 400\n",
      "Currently processing 209 / 400\n",
      "Currently processing 210 / 400\n",
      "Currently processing 211 / 400\n",
      "Currently processing 212 / 400\n",
      "Currently processing 213 / 400\n",
      "Currently processing 214 / 400\n",
      "Currently processing 215 / 400\n",
      "Currently processing 216 / 400\n",
      "Currently processing 217 / 400\n",
      "Currently processing 218 / 400\n",
      "Currently processing 219 / 400\n",
      "Currently processing 220 / 400\n",
      "Currently processing 221 / 400\n",
      "Currently processing 222 / 400\n",
      "Currently processing 223 / 400\n",
      "Currently processing 224 / 400\n",
      "Currently processing 225 / 400\n",
      "Currently processing 226 / 400\n",
      "Currently processing 227 / 400\n",
      "Currently processing 228 / 400\n",
      "Currently processing 229 / 400\n",
      "Currently processing 230 / 400\n",
      "Currently processing 231 / 400\n",
      "Currently processing 232 / 400\n",
      "Currently processing 233 / 400\n",
      "Currently processing 234 / 400\n",
      "Currently processing 235 / 400\n",
      "Currently processing 236 / 400\n",
      "Currently processing 237 / 400\n",
      "Currently processing 238 / 400\n",
      "Currently processing 239 / 400\n",
      "Currently processing 240 / 400\n",
      "Currently processing 241 / 400\n",
      "Currently processing 242 / 400\n",
      "Currently processing 243 / 400\n",
      "Currently processing 244 / 400\n",
      "Currently processing 245 / 400\n",
      "Currently processing 246 / 400\n",
      "Currently processing 247 / 400\n",
      "Currently processing 248 / 400\n",
      "Currently processing 249 / 400\n",
      "Currently processing 250 / 400\n",
      "Currently processing 251 / 400\n",
      "Currently processing 252 / 400\n",
      "Currently processing 253 / 400\n",
      "Currently processing 254 / 400\n",
      "Currently processing 255 / 400\n",
      "Currently processing 256 / 400\n",
      "Currently processing 257 / 400\n",
      "Currently processing 258 / 400\n",
      "Currently processing 259 / 400\n",
      "Currently processing 260 / 400\n",
      "Currently processing 261 / 400\n",
      "Currently processing 262 / 400\n",
      "Currently processing 263 / 400\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Currently processing 264 / 400\n",
      "Currently processing 265 / 400\n",
      "Currently processing 266 / 400\n",
      "Currently processing 267 / 400\n",
      "Currently processing 268 / 400\n",
      "Currently processing 269 / 400\n",
      "Currently processing 270 / 400\n",
      "Currently processing 271 / 400\n",
      "Currently processing 272 / 400\n",
      "Currently processing 273 / 400\n",
      "Currently processing 274 / 400\n",
      "Currently processing 275 / 400\n",
      "Currently processing 276 / 400\n",
      "Currently processing 277 / 400\n",
      "Currently processing 278 / 400\n",
      "Currently processing 279 / 400\n",
      "Currently processing 280 / 400\n",
      "Currently processing 281 / 400\n",
      "Currently processing 282 / 400\n",
      "Currently processing 283 / 400\n",
      "Currently processing 284 / 400\n",
      "Currently processing 285 / 400\n",
      "Currently processing 286 / 400\n",
      "Currently processing 287 / 400\n",
      "Currently processing 288 / 400\n",
      "Currently processing 289 / 400\n",
      "Currently processing 290 / 400\n",
      "Currently processing 291 / 400\n",
      "Currently processing 292 / 400\n",
      "Currently processing 293 / 400\n",
      "Currently processing 294 / 400\n",
      "Currently processing 295 / 400\n",
      "Currently processing 296 / 400\n",
      "Currently processing 297 / 400\n",
      "Currently processing 298 / 400\n",
      "Currently processing 299 / 400\n",
      "Currently processing 300 / 400\n",
      "Cost and accuracy (dev) 0.3673995260534299 0.87\n",
      "Currently processing 301 / 400\n",
      "Currently processing 302 / 400\n",
      "Currently processing 303 / 400\n",
      "Currently processing 304 / 400\n",
      "Currently processing 305 / 400\n",
      "Currently processing 306 / 400\n",
      "Currently processing 307 / 400\n",
      "Currently processing 308 / 400\n",
      "Currently processing 309 / 400\n",
      "Currently processing 310 / 400\n",
      "Currently processing 311 / 400\n",
      "Currently processing 312 / 400\n",
      "Currently processing 313 / 400\n",
      "Currently processing 314 / 400\n",
      "Currently processing 315 / 400\n",
      "Currently processing 316 / 400\n",
      "Currently processing 317 / 400\n",
      "Currently processing 318 / 400\n",
      "Currently processing 319 / 400\n",
      "Currently processing 320 / 400\n",
      "Currently processing 321 / 400\n",
      "Currently processing 322 / 400\n",
      "Currently processing 323 / 400\n",
      "Currently processing 324 / 400\n",
      "Currently processing 325 / 400\n",
      "Currently processing 326 / 400\n",
      "Currently processing 327 / 400\n",
      "Currently processing 328 / 400\n",
      "Currently processing 329 / 400\n",
      "Currently processing 330 / 400\n",
      "Currently processing 331 / 400\n",
      "Currently processing 332 / 400\n",
      "Currently processing 333 / 400\n",
      "Currently processing 334 / 400\n",
      "Currently processing 335 / 400\n",
      "Currently processing 336 / 400\n",
      "Currently processing 337 / 400\n",
      "Currently processing 338 / 400\n",
      "Currently processing 339 / 400\n",
      "Currently processing 340 / 400\n",
      "Currently processing 341 / 400\n",
      "Currently processing 342 / 400\n",
      "Currently processing 343 / 400\n",
      "Currently processing 344 / 400\n",
      "Currently processing 345 / 400\n",
      "Currently processing 346 / 400\n",
      "Currently processing 347 / 400\n",
      "Currently processing 348 / 400\n",
      "Currently processing 349 / 400\n",
      "Currently processing 350 / 400\n",
      "Currently processing 351 / 400\n",
      "Currently processing 352 / 400\n",
      "Currently processing 353 / 400\n",
      "Currently processing 354 / 400\n",
      "Currently processing 355 / 400\n",
      "Currently processing 356 / 400\n",
      "Currently processing 357 / 400\n",
      "Currently processing 358 / 400\n",
      "Currently processing 359 / 400\n",
      "Currently processing 360 / 400\n",
      "Currently processing 361 / 400\n",
      "Currently processing 362 / 400\n",
      "Currently processing 363 / 400\n",
      "Currently processing 364 / 400\n",
      "Currently processing 365 / 400\n",
      "Currently processing 366 / 400\n",
      "Currently processing 367 / 400\n",
      "Currently processing 368 / 400\n",
      "Currently processing 369 / 400\n",
      "Currently processing 370 / 400\n",
      "Currently processing 371 / 400\n",
      "Currently processing 372 / 400\n",
      "Currently processing 373 / 400\n",
      "Currently processing 374 / 400\n",
      "Currently processing 375 / 400\n",
      "Currently processing 376 / 400\n",
      "Currently processing 377 / 400\n",
      "Currently processing 378 / 400\n",
      "Currently processing 379 / 400\n",
      "Currently processing 380 / 400\n",
      "Currently processing 381 / 400\n",
      "Currently processing 382 / 400\n",
      "Currently processing 383 / 400\n",
      "Currently processing 384 / 400\n",
      "Currently processing 385 / 400\n",
      "Currently processing 386 / 400\n",
      "Currently processing 387 / 400\n",
      "Currently processing 388 / 400\n",
      "Currently processing 389 / 400\n",
      "Currently processing 390 / 400\n",
      "Currently processing 391 / 400\n",
      "Currently processing 392 / 400\n",
      "Currently processing 393 / 400\n",
      "Currently processing 394 / 400\n",
      "Currently processing 395 / 400\n",
      "Currently processing 396 / 400\n",
      "Currently processing 397 / 400\n",
      "Currently processing 398 / 400\n",
      "Currently processing 399 / 400\n",
      "Cost and accuracy (dev) 0.2648193019230353 0.9075\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3xcdbnv8c+TSZppm7bpJQV6gRSsCLJpaWNP3cilqFAVi4C1VVHwbAS5ibJVLlsFBM5h42WzUbcIiAeh0nJTWkTYIFc3Ukmk3CwoLSqhXEJL0qRtLpN5zh9rJZlMJskkzVwy832/XnllzazfmvXk1876zvqttWaZuyMiIpKsJNcFiIhIflJAiIhISgoIERFJSQEhIiIpKSBERCQlBYSIiKSkgBABzCxiZi1mtvdIthUZzUzXQchoZGYtCQ/HAW1AZ/j4dHdflf2qRAqLAkJGPTP7G3Cquz84QJtSd49lr6rcKra/VzJDQ0xSkMzscjNbY2a3mlkzcJKZvd/MnjSzRjN73cyuMbOysH2pmbmZVYePbwnn/9bMms3sD2Y2Z6htw/kfMbO/mFmTmf3QzP7HzE7pp+5SM/uWmW0ys+1mVmtmM8zsXWbmSW1/3/U6ZnaqmT0W1rENuCJc/j0J7fc0s11mNjV8vMzMngn74/dmdtDI9L4UCgWEFLLjgV8Ck4A1QAw4F5gGHAosBU4fYPnPAN8CpgD/AC4balszmw7cBnw9XO8rwKIBXufrwCfD2iqBU4HWAdon+mdgI1AFXAL8Gvh0wvwVwO/cfauZvQ+4Pnz9qcCNwN1mNibNdUkRUEBIIfu9u69z97i773L3p9x9vbvH3H0zcB1wxADL3+Hute7eAawC5g+j7bHABne/O5z3H8DbA7zOqcBF7v7XsO4N7r4tzb/3H+7+E3fvdPddBOGYGBCfCZ8DOA34r7BPOt39xvD596W5LikCpbkuQCSDXk18EA63fB9YSHBguxRYP8DybyRM7wQqhtF2RmId7u5mVj/A68wGNg0wfyCvJj1+EKg0s4VAI/Be4O5w3j7AZ83sqwntxwAzh7luKUDag5BClnwGxk+B54F3uftE4NuAZbiG14FZXQ/MzBh4I/wqsF+K53eEy49LeG7PpDa9/t7wIPXtBHsRnwHudvcdCeu51N0rE37GufttafxNUiQUEFJMJgBNwA4zO4CBjz+MlHuABWb2cTMrJTgGUjVA+xuAy81sPwvMN7MpBHsobxAcbI+Y2WkEewGD+SXBsYfE4SUIhtfOMrP3heupCGscP4y/UQqUAkKKyb8CJwPNBHsTazK9Qnd/k2AD/QNgK8HewdME122k8l2Cg8u/A7YTbMijHpyP/kXgIoJjGO9i4OGxLk8QHJyvAv47oa71wBnAT4B3gL8AJw3tr5NCp+sgRLLIzCLAFuCT7v54rusRGYj2IEQyzMyWmtkkMysnOBU2Bvwxx2WJDEoBIZJ5HwA2EwwNLQU+4e79DTGJ5A0NMYmISEragxARkZQK5kK5adOmeXV1da7LEBEZVerq6t5295SnXhdMQFRXV1NbW5vrMkRERhUz+3t/8zTEJCIiKSkggJ3t+tp8EZFkRR8Qr27byQe//yi3PZX8PWciIsWtYI5BDFfVhHLevccEzr/rWQA+9b7ZOa5IRNLR0dFBfX09ra3p3i6juEWjUWbNmkVZWVnayxR9QETLIvz0cws5/eY6hYTIKFJfX8+ECROorq4m+JJc6Y+7s3XrVurr65kzZ87gC4SKfogJekLi8LlVnH/XsxpuEhkFWltbmTp1qsIhDWbG1KlTh7y3pYAIKSRERh+FQ/qG01cKiAQKCRGRHgqIJAoJERmOSy65hO9973t58zojQQGRgkJCREQB0S+FhIgM5oorrmD//ffnQx/6EC+99FL385s2bWLp0qUsXLiQww47jBdffJGmpiaqq6uJx+MA7Ny5k9mzZ9PR0dHv62/YsIHFixdz8MEHc/zxx/POO+8AcM0113DggQdy8MEHs3LlSgAeffRR5s+fz/z58znkkENobm7e7b+v6E9zHYhOgRUZHS5d9wJ/3rJ9RF/zwBkTufjj7+13fl1dHatXr+bpp58mFouxYMECFi5cCMBpp53Gtddey9y5c1m/fj1nnnkmDz30EPPmzePRRx9lyZIlrFu3jmOOOWbA6xI+//nP88Mf/pAjjjiCb3/721x66aVcffXVXHnllbzyyiuUl5fT2NgIwPe+9z1+/OMfc+ihh9LS0kI0Gt3tPtAexCC0JyEiqTz++OMcf/zxjBs3jokTJ7Js2TIAWlpaeOKJJ1i+fDnz58/n9NNP5/XXXwdgxYoVrFkT3Ap99erVrFixot/Xb2pqorGxkSOOOAKAk08+mcceewyAgw8+mM9+9rPccsstlJYGn/MPPfRQzjvvPK655hoaGxu7n98d2oNIg/YkRPLbQJ/0MynVqaPxeJzKyko2bNjQZ96yZcu48MIL2bZtG3V1dRx11FHDWu9vfvMbHnvsMdauXctll13GCy+8wAUXXMDHPvYx7r33XhYvXsyDDz7Ie97znmG9fhftQaRJexIikujwww/nV7/6Fbt27aK5uZl169YBMHHiRObMmcPtt98OBFcxP/PMMwBUVFSwaNEizj33XI499lgikUi/rz9p0iQmT57M448/DsDNN9/MEUccQTwe59VXX2XJkiVcddVVNDY20tLSwqZNm/inf/onzj//fGpqanjxxRd3+2/UHsQQaE9CRLosWLCAFStWMH/+fPbZZx8OO+yw7nmrVq3ijDPO4PLLL6ejo4OVK1cyb948IBhmWr58OY888sig67jpppv40pe+xM6dO9l33335+c9/TmdnJyeddBJNTU24O1/96leprKzkW9/6Fg8//DCRSIQDDzyQj3zkI7v9NxbMPalramo8WzcMau3o5PSb63jsrw38+wkHKyREcmDjxo0ccMABuS5jVEnVZ2ZW5+41qdpriGkYNNwkIsVAATFMCgkRKXQKiN2gkBDJrUIZIs+G4fSVAmI3JYbEN+58ljVP/SPXJYkUhWg0ytatWxUSaei6H8RQL57TWUwjoNfZTXc+B8CK9+2d46pECtusWbOor6+noaEh16WMCl13lBsKBcQIUUiIZFdZWdmQ7o4mQ6chphHUFRJHvLuK8+98TsNNIjKqKSBGmEJCRAqFAiIDFBIiUgjyNiDMbLaZPWxmG83sBTM7N9c1DYVCQkRGu7wNCCAG/Ku7HwAsBs4yswNzXNOQKCREZDTL24Bw99fd/U/hdDOwEZiZ26qGriskjtxfISEio0veBkQiM6sGDgHWJz1/mpnVmlltPp8LHS2LcO1JCgkRGV3yPiDMrAK4E/iKu/e6p6C7X+fuNe5eU1VVlZsC06SQEJHRJq8DwszKCMJhlbvflet6dpdCQkRGk7wNCAvu5fczYKO7/yDX9YwUhYSIjBZ5GxDAocDngKPMbEP489FcFzUSFBIiMhrk7Xcxufvvgb53BC8QXSHxpVv03U0ikp/yeQ+i4GlPQkTymQIixxQSIpKvFBB5QCEhIvlIAZEnFBIikm8UEHlEISEi+UQBkWeSQ2L1HxUSIpIbWQkIMzvXzCZa4Gdm9iczOzob6x6NEkPigrsUEiKSG9nag/jf4fcoHQ1UAV8ArszSukclhYSI5Fq2AqLrgrePAj9392co4IvgRopCQkRyKVsBUWdm/00QEPeb2QQgnqV1j2oKCRHJlWwFxL8AFwDvc/edQBnBMJOkQSEhIrmQrYB4P/CSuzea2UnAN4GmLK27ICgkRCTbshUQPwF2mtk84BvA34FfZGndBUMhISLZlK2AiLm7A8cB/+nu/wlMyNK6C4pCQkSyJVsB0WxmFxLc3+E3ZhYhOA4hw6CQEJFsyFZArADaCK6HeAOYCXw3S+suSAoJEcm0rAREGAqrgElmdizQ6u46BrGbFBIikknZ+qqNTwF/BJYDnwLWm9kns7HuQqeQEJFMydYtR/+N4BqItwDMrAp4ELgjS+svaF0hccYtdVxwV3D70pWLdPtSEdk92ToGUdIVDqGtWVx3UYiWRfjJSQtZoj0JERkh2dpI32dm95vZKWZ2CvAb4N4srbtoKCREZCRl6yD114HrgIOBecB17n5+NtZdbBQSIjJSsnUMAne/E7gzW+srZl0hoWMSIrI7MroHYWbNZrY9xU+zmW3P5LqLnfYkRGR3ZTQg3H2Cu09M8TPB3Sdmct3SNyRuVUiIyBDoTKIClxgSFyokRGQIFBBFQCEhIsOhgCgSCgkRGaq8DQgzu9HM3jKz53NdS6FQSIjIUORtQAD/D1ia6yIKjUJCRNKVtwHh7o8B23JdRyFSSIhIOvI2INJhZqeZWa2Z1TY0NOS6nFFFISEigxnVAeHu17l7jbvXVFVV5bqcUUchISIDGdUBIbtPISEi/VFAiEJCRFLK24Aws1uBPwD7m1m9mf1LrmsqZAoJEUmWtW9zHSp3/3Suayg2id8Ce2H4LbCf1rfAihStvN2DkNzQnoSIdFFASB8KCREBBYT0QyEhIgoI6ZdCQqS4KSBkQAoJkeKlgJBBKSREipO5e65rGBE1NTVeW1ub6zIKWmtHJ2fcUsfDLzWwqHoKe0yKMn1COXtMLGf6hGB6+sQo0yeWM6G8FDPLdckiMggzq3P3mlTz8vY6CMk/0bII135uIf/+25d4/rUmnqtv5M3tbezq6OzTdmxZhOkTy9ljQpSqieVhkER7/Z4+McrEqIJEJF8pIGRIyksjfPvjB3Y/dnda2mK81dzGm9tbaQh/v7W9rfu5jVu288j2Vna09w2S8tKS7iCZ3rUnkvR4j4nlTBpbpiARyTIFhOwWM2NCtIwJ0TL2q6oYsG1LW4y3trfyVnMQHl3TXYHy0hvNPP6Xt2lui/VZdkxpSbDXkbQH0vW7a5hr8jgFichIUUBI1lSUl1JRVcG+gwTJzvZYrz2QIFC69kpa+etbLfzPy2+zvbVvkJRFjOkTolQlHBvp+l2VsGcyZdwYSkoUJCIDUUBI3hk3ppTqaaVUTxs/YLvWjs7u0Hgz6XdDcxuvvL2D9a9so3FnR59lS0uMqsS9kKRjJMG8cqaOLyeiIJEipYCQUStaFmHvqePYe+q4Adu1dnTS0GsvpGfP5M3trby6bSe1f9vGOymCJFJiTKsY03dYK2HPZI+J5UytUJBI4VFASMGLlkWYPWUcs6cMHCTtsTgNLYkH2Vt77aG81tjK0/9oZOuO9j7LlhhMqyjvdWC9KiFEJkRLiZZFKC8toby0pGc6/F0W0SVJkn8UECKhMaUlzKwcy8zKsQO2a4/Febul50D7m81tNGzvGd56o6mVZ+ub2LqjjXQvM4qUWN/wKI1QXlZCNPydGCjlpRGiZWGb0pKkdr3n9YRR6nna85H+KCBEhmhMaQkzKscyY5AgiXXGebulnbeaW2lpi9HWEact1klbLE5rR/C7rSNhOtZJa0Kbto44rbFO2jri7GiLsbUlcfme6fZYfLf+ntKucCqLEE0MoVR7PClCq9e8pEAaaF55qcIp3ykgRDKkNFLCnpOi7DkpmtH1xONOe2e8O4BaUwVR4vMd8QHntcbitCXMa26N8XZLe8Kynd3h1dG5e9/EUBaxPns1Y0pLGDcmwvjyUsaPKQ1+lwePK8pLk+ZFwueCeV3tyktLdLrzCFBAiIxyJSVGtCRCtCwClGV13Z1xpz3F3s9Q95AS57WGyzS3xnhzeys72jppaYuxoy1GLJ5eIEVKjHFjImFolDK+K1T6TPeEShBAke6wSVx+bFmkKE+LVkCIyLBFSoyxYyKMHRPJyvraYp3saOtkR1uMHe2xnum2GDvaE5+PpWy3bcdOdobtWtpitKU5PGcG48oSgiUhSPoPoAjju8ImDJ/x4d7O+DERSkfBiQkKCBEZNYJhqAhTxo8ZkdeLdcbZ0d7JzjBUWto62RmGx872nj2X7vBJmn6ruZUdb/eel/7fUtIdML32dsJgSRxa6348pndI9QRQ0C8jTQEhIkWrNFLCpLElTBo7MkNz8bizq6MzRaj0Dp8dbUEoJQdR464OXmvc1dOuvZPONIbV5s2axN1nf2BE/oZECggRkRFSUmLdn/BHgrvTFgvOYku1R9PSFmNnW4zKcSOzR5VMASEikqfMjGhZcALC1BysP/+PkoiISE4oIEREJKWCueWomTUAf9+Nl5gGvD1C5Ywk1TU0qmtoVNfQFGJd+7h7VaoZBRMQu8vMavu7L2suqa6hUV1Do7qGptjq0hCTiIikpIAQEZGUFBA9rst1Af1QXUOjuoZGdQ1NUdWlYxAiIpKS9iBERCQlBYSIiKRUVAFhZkvN7CUze9nMLkgxv9zM1oTz15tZdZ7UdYqZNZjZhvDn1CzVdaOZvWVmz/cz38zsmrDuZ81sQZ7UdaSZNSX017ezVNdsM3vYzDaa2Qtmdm6KNlnvszTrynqfmVnUzP5oZs+EdV2aok3W35Np1pWT92S47oiZPW1m96SYN7L95e5F8QNEgE3AvsAY4BngwKQ2ZwLXhtMrgTV5UtcpwI9y0GeHAwuA5/uZ/1Hgt4ABi4H1eVLXkcA9OeivvYAF4fQE4C8p/i2z3mdp1pX1Pgv7oCKcLgPWA4uT2uTiPZlOXTl5T4brPg/4Zap/r5Hur2Lag1gEvOzum929HVgNHJfU5jjgpnD6DuCDlvn7FqZTV064+2PAtgGaHAf8wgNPApVmtlce1JUT7v66u/8pnG4GNgIzk5plvc/SrCvrwj5oCR+WhT/JZ81k/T2ZZl05YWazgI8BN/TTZET7q5gCYibwasLjevq+SbrbuHsMaIKMf4liOnUBnBgOSdxhZrMzXFO60q09F94fDhH81szem+2Vh7v2hxB8+kyU0z4boC7IQZ+FwyUbgLeAB9y93/7K4nsynbogN+/Jq4FvAP3dCm9E+6uYAiJViiZ/KkinzUhLZ53rgGp3Pxh4kJ5PCLmWi/5Kx58Ivl9mHvBD4NfZXLmZVQB3Al9x9+3Js1MskpU+G6SunPSZu3e6+3xgFrDIzA5KapKT/kqjrqy/J83sWOAtd68bqFmK54bdX8UUEPVAYsrPArb018bMSoFJZH4oY9C63H2ru7eFD68HFma4pnSl06dZ5+7bu4YI3P1eoMzMpmVj3WZWRrARXuXud6VokpM+G6yuXPZZuM5G4BFgadKsXLwnB60rR+/JQ4FlZvY3gqHoo8zslqQ2I9pfxRQQTwFzzWyOmY0hOICzNqnNWuDkcPqTwEMeHu3JZV1JY9TLCMaQ88Fa4PPhmTmLgSZ3fz3XRZnZnl3jrma2iOD/+dYsrNeAnwEb3f0H/TTLep+lU1cu+szMqsysMpweC3wIeDGpWdbfk+nUlYv3pLtf6O6z3L2aYDvxkLuflNRsRPuraO4o5+4xMzsbuJ/gzKEb3f0FM/sOUOvuawneRDeb2csEqbsyT+r6spktA2JhXadkui4AM7uV4OyWaWZWD1xMcMAOd78WuJfgrJyXgZ3AF/Kkrk8CZ5hZDNgFrMxC0EPwCe9zwHPh+DXARcDeCbXlos/SqSsXfbYXcJOZRQgC6TZ3vyfX78k068rJezKVTPaXvmpDRERSKqYhJhERGQIFhIiIpKSAEBGRlArmIPW0adO8uro612WIiIwqdXV1b3s/96QumICorq6mtrY212WIiIwqZvb3/uZpiElERFIqmD0IEZFC1xl3drTH2NEWo6U1Rktb8FNeGmHRnCkjvj4FhIhIBsXjzs6OTna0xWhuDTfuXT8JG/mWtp4Nf3Nb33Y72mLsaO9MuY55syu5+6xDR7z2gg6Ijo4O6uvraW1tzXUpeS0ajTJr1izKyspyXYpIXnB3dnV0JmycO2lu6wim27s27J20tHUE81pjPdMJG/odbTFa2mOkcz1yWcSoKC9lfHkpFeWlTIiWMmX8GPaeMo6K8Lnx4fPd7cLpKePHZKQfCjog6uvrmTBhAtXV1WT+tg6jk7uzdetW6uvrmTNnTq7LERk2d6ctFu/7yTzcqA/103s8jY16pMS6N97BRjvCpLFlzKocy/jyCBXlZVSUR8INeRnjyyNMiJYyfkzPxr0i3NCXl0Yy30lDVNAB0draqnAYhJkxdepUGhoacl2KjBLuTtyD8fC4e/fveBw6vWva6Qznedi2M7xLWWe8Z9ley3e9ZtxpjXUGn9DDjXpz4ifypI16c2vPdCyNrboZSRv14FP5nhOj3Z/euzba48tLmZDULvETfnlpSUFvXwo6IICC/scbKeqjkeXubN8V47XGXWxp3MWWpl3saOvsteGMOz3T3Rta+mw0g9/0XS5pA9vphBvfhI2yJ7ejZ12JtYTr7V5Xr41+Uhv3tIZLMqHrE3riBnzq+HFURIONeOKQS0U/G/rx5aWMGxPR//k0FXxAiIy0tlgnbza19QRAGAKvNbaypXEXrzfu6vdgYqISC4YoSiz4CaYTnisxIhY8V1ISzI+YYamW61q2a7kSo6wk8XW7poNlzYLXCqbpnk5+nd7L96y3q9aeGrvqTXj9pHqT6xio/mhZJBiaCYdlxo8ppaREG/VsU0Bk0SWXXEJFRQVf+9rXcl2K9MPd2bqjPdzwt6YMgIbmtj7LTasoZ0ZllHdVVXD43CpmVEaZWTmWGZVj2asyyoTysp4NZbgRFMl3CggpKrvaO9nStCv8pN/aaxioKxDaYr1v9xstK+ne2L9n/+nMqBzbKwD2nBQlWpZ/BxhFdlfRBMSl617gz1uSb8O7ew6cMZGLPz7wvd2vuOIKfvGLXzB79myqqqpYuDC4M+GmTZs466yzaGhoYNy4cVx//fXstddezJs3j82bN1NSUsLOnTvZf//92bx5c69TUNetW8fll19Oe3s7U6dOZdWqVeyxxx60tLRwzjnnUFtbi5lx8cUXc+KJJ3Lfffdx0UUX0dnZybRp0/jd7343ov2QL+Jxp6Elaeinay8gDIBtO9p7LWMGe0yIMqMyyoEzJvLhA/dgxqRoGAJjmVk5lspxZRqzlqJUNAGRC3V1daxevZqnn36aWCzGggULugPitNNO49prr2Xu3LmsX7+eM888k4ceeoh58+bx6KOPsmTJEtatW8cxxxzT5/qED3zgAzz55JOYGTfccANXXXUV3//+97nsssuYNGkSzz33HADvvPMODQ0NfPGLX+Sxxx5jzpw5bNuWtdv5jriWthhbGnf1GwBvNLXS0dn7CGpFeWn4ST/KvFmV3Z/+Z0zq+fRfFtE3zoikUjQBMdgn/Ux4/PHHOf744xk3bhwAy5YtA6ClpYUnnniC5cuXd7dtawvGtVesWMGaNWtYsmQJq1ev5swzz+zzuvX19axYsYLXX3+d9vb27usXHnzwQVavXt3dbvLkyaxbt47DDz+8u82UKSN/Of5IiHXGebO5rXvD/1pyADTuYntrrNcykRJjz4nBUM/CvSeH4/1jmVnZswcwMaqL/0SGq2gCIldSDU3E43EqKyvZsGFDn3nLli3jwgsvZNu2bdTV1XHUUUf1aXPOOedw3nnnsWzZMh555BEuueQSIDjAmry+VM9lm7vTtKuD18Jx/+CAb++N/5vbW/tcmFQ5rowZk8Yya/I4/tecKX0CYPqEKBEd7BXJGAVEBh1++OGccsopXHDBBcRiMdatW8fpp5/OxIkTmTNnDrfffjvLly/H3Xn22WeZN28eFRUVLFq0iHPPPZdjjz2WSKTvwc+mpiZmzpwJwE033dT9/NFHH82PfvQjrr76aiAYYnr/+9/PWWedxSuvvNI9xDTSexFtsU7eaGrtCYCks362NO5iZ9Jpn2MiJewVDvX8837Tujf6XQGw16SxjC/Xf0+RXNI7MIMWLFjAihUrmD9/Pvvssw+HHXZY97xVq1ZxxhlncPnll9PR0cHKlSuZN28eEAwzLV++nEceeSTl615yySUsX76cmTNnsnjxYl555RUAvvnNb3LWWWdx0EEHEYlEuPjiiznhhBO47rrrOOGEE4jH40yfPp0HHnhg2H9TPO5c//hmnqlvHOS0zzHMqBzb72mf08aX61RPkTxnnsHLIs1sKfCfQAS4wd2vTJr/H8CS8OE4YLq7V4bzOoHnwnn/cPdlA62rpqbGk28YtHHjRg444IDd/juKQTp9FY8759/5LLfX1TNn2nhmTR7bfbC3KwD2qhzLXjrtU2TUMLM6d69JNS9jexBmFgF+DHwYqAeeMrO17v7nrjbu/tWE9ucAhyS8xC53n5+p+mRoEsPh3A/O5asffneuSxKRDMvk+X2LgJfdfbO7twOrgeMGaP9p4NYM1iPDpHAQKU6ZDIiZwKsJj+vD5/ows32AOcBDCU9HzazWzJ40s0/0s9xpYZva/r6NNJNDaIVioD5SOIgUr0wGRKojkP1tiVYCd7h74qkue4fjYp8Brjaz/fq8mPt17l7j7jVVVVV9XjQajbJ161aFxAC67gcRjUb7zFM4iBS3TJ7FVA/MTng8C9jST9uVwFmJT7j7lvD3ZjN7hOD4xKahFDBr1izq6+t1r4NBdN1RLpHCQUQyGRBPAXPNbA7wGkEIfCa5kZntD0wG/pDw3GRgp7u3mdk04FDgqqEWUFZWprukDYPCQUQggwHh7jEzOxu4n+A01xvd/QUz+w5Q6+5rw6afBlZ773GgA4CfmlmcYBjsysSznyRzFA4i0iWj10FkU6rrIGRoFA4ixWeg6yD0NZYCKBxEpC8FhCgcRCQlBUSRUziISH/SCggzu9PMPmZmCpQConAQkYGku8H/CcEpqn81syvN7D0ZrEmyQOEgIoNJKyDc/UF3/yywAPgb8ICZPWFmXzAz3bJrlInHnQvuUjiIyMDSHjIys6nAKcCpwNMEX+O9ABj+zQUk67rC4bZahYOIDCytC+XM7C7gPcDNwMfd/fVw1hoz08UHo0RiOHxZ4SAig0j3SuofuftDqWb0d4GF5Jc+4fChubkuSUTyXLpDTAeYWWXXAzObbGZnZqgmGWGpwsFMt/sUkYGlGxBfdPfGrgfu/g7wxcyUJCNJ4SAiw5VuQJRYwlYlvJ3omMyUJCNF4SAiuyPdYxD3A7eZ2bUEN/35EnBfxqqS3RaPOxfe9ZzCQUSGLd2AOB84HTiD4E5x/w3ckKmiZGI+U78AAAxASURBVPd0hcOa2lcVDiIybGkFhLvHCa6m/klmy5Hd1SscjnqXwkFEhi3d6yDmAv8XOBDovnmxu++bobpkGPqEw4ffrXAQkWFL9yD1zwn2HmLAEuAXBBfNSZ5QOIjISEs3IMa6++8I7kD3d3e/BDgqc2XJUCgcRCQT0g2I1vCrvv9qZmeb2fHA9MEWMrOlZvaSmb1sZhekmH+KmTWY2Ybw59SEeSeb2V/Dn5PT/ouKjMJBRDIl3bOYvgKMA74MXEYwzDTgRju8VuLHwIeBeuApM1vr7n9OarrG3c9OWnYKcDFQQ3BabV247Dtp1lsU4nHnol8pHEQkMwbdgwg39J9y9xZ3r3f3L7j7ie7+5CCLLgJedvfN7t4OrAaOS7OuY4AH3H1bGAoPAEvTXLYodIXD6qcUDiKSGYMGhLt3Agtt6FufmcCrCY/rw+eSnWhmz5rZHWY2eyjLmtlpZlZrZrUNDQ1DLG/0UjiISDakewziaeBuM/ucmZ3Q9TPIMqm2WJ70eB1Q7e4HAw8CNw1hWdz9OnevcfeaqqqqQcopDInhcI7CQUQyKN1jEFOArfQ+c8mBuwZYph6YnfB4FrAlsYG7b014eD3w7wnLHpm07CNp1lqwksPhPIWDiGRQuldSf2EYr/0UMNfM5gCvASsJ7mvdzcz2Srj50DJgYzh9P/B/zGxy+Pho4MJh1FAwFA4ikm3pXkn9c1IP8fzv/pZx95iZnU2wsY8AN7r7C2b2HaDW3dcCXzazZQQX4G0juKUp7r7NzC4jCBmA77j7tvT/rMKicBCRXDD3Ptv9vo3MTkx4GAWOB7a4+5czVdhQ1dTUeG1t4d39NB53/u3Xz3HrHxUOIjLyzKyuvzuDpjvEdGfSC95KcFBZMkjhICK5lO5ZTMnmAnuPZCHSm8JBRHIt3WMQzfQ+BvEGwT0iJAMUDiKSD9IdYpqQ6UIkkBgOZy9ROIhI7qQ1xGRmx5vZpITHlWb2icyVVZySw+Ffj1Y4iEjupHsM4mJ3b+p64O6NBF+mJyNE4SAi+SbdgEjVLt2rsGUQQTg8r3AQkbySbkDUmtkPzGw/M9vXzP4DqMtkYcWiJxz+oXAQkbySbkCcA7QDa4DbgF3AWZkqqlgoHEQkn6V7FtMOoM8d4WT4FA4iku/SPYvpATOrTHg82czuz1xZhS0xHM5asp/CQUTyUrpDTNPCM5cACO/yNug9qaWv5HD42tH7KxxEJC+lGxBxM+v+ag0zqybFt7vKwBQOIjKapHuq6r8BvzezR8PHhwOnZaakwhSPO9+8W+EgIqNHugep7zOzGoJQ2ADcTXAmk6ShKxx+uV7hICKjR7pf1ncqcC7BrT83AIuBP9D7FqSSgsJBREardI9BnAu8D/i7uy8BDgEaMlZVgVA4iMholm5AtLp7K4CZlbv7i8D+mStr9FM4iMhol25A1IfXQfwaeMDM7ga2DLaQmS01s5fM7GUz63OhnZmdZ2Z/NrNnzex3ZrZPwrxOM9sQ/qxN9w/KB4nhcOaRCgcRGZ3SPUh9fDh5iZk9DEwC7htoGTOLAD8GPgzUA0+Z2Vp3/3NCs6eBGnffaWZnAFcBK8J5u9x9fvp/Sn5IDoevH6NwEJHRacjfyOrujw7eCoBFwMvuvhnAzFYDxwHdAeHuDye0fxI4aaj15JN43PmWwkFECsRw70mdjpnAqwmP68Pn+vMvwG8THkfNrNbMnuzv5kRmdlrYprahIbfHzLvCYZXCQUQKRCbv6ZBq65jy6mszOwmoAY5IeHpvd99iZvsCD5nZc+6+qdeLuV8HXAdQU1OTsyu7FQ4iUogyuQdRD8xOeDyLFAe2zexDBFdqL3P3tq7n3X1L+Hsz8AjBqbV5R+EgIoUqkwHxFDDXzOaY2RhgJdDrbCQzOwT4KUE4vJXw/GQzKw+npwGHknDsIl8oHESkkGVsiMndY2Z2NnA/EAFudPcXzOw7QK27rwW+C1QAt4cb1n+4+zLgAOCnZhYnCLErk85+yjmFg4gUOnMvjC9lramp8dra2qysKzEczjhyP76hcBCRUcrM6ty9JtW8TA4xFaR43Pn2WoWDiBQ+BcQQdIXDLU8qHESk8Ckg0qRwEJFio4BIg8JBRIqRAmIQCgcRKVYKiAEoHESkmCkg+uHeEw5fOkLhICLFRwGRgntwnUNXOJy/VOEgIsVHAZFE4SAiElBAJFA4iIj0UECEFA4iIr0pIFA4iIikUvQBoXAQEUmt6ANi89s7uLPuNYWDiEiSTN5ydFTYr6qC+75yGHtPGadwEBFJUPQBAbDP1PG5LkFEJO8U/RCTiIikpoAQEZGUCuaWo2bWAPx9N15iGvD2CJUzklTX0KiuoVFdQ1OIde3j7lWpZhRMQOwuM6vt776suaS6hkZ1DY3qGppiq0tDTCIikpICQkREUlJA9Lgu1wX0Q3UNjeoaGtU1NEVVl45BiIhIStqDEBGRlBQQIiKSUlEFhJktNbOXzOxlM7sgxfxyM1sTzl9vZtV5UtcpZtZgZhvCn1OzVNeNZvaWmT3fz3wzs2vCup81swV5UteRZtaU0F/fzlJds83sYTPbaGYvmNm5Kdpkvc/SrCvrfWZmUTP7o5k9E9Z1aYo2WX9PpllXTt6T4bojZva0md2TYt7I9pe7F8UPEAE2AfsCY4BngAOT2pwJXBtOrwTW5EldpwA/ykGfHQ4sAJ7vZ/5Hgd8CBiwG1udJXUcC9+Sgv/YCFoTTE4C/pPi3zHqfpVlX1vss7IOKcLoMWA8sTmqTi/dkOnXl5D0Zrvs84Jep/r1Gur+KaQ9iEfCyu29293ZgNXBcUpvjgJvC6TuAD1rmv+I1nbpywt0fA7YN0OQ44BceeBKoNLO98qCunHD31939T+F0M7ARmJnULOt9lmZdWRf2QUv4sCz8ST5rJuvvyTTrygkzmwV8DLihnyYj2l/FFBAzgVcTHtfT903S3cbdY0ATMDUP6gI4MRySuMPMZme4pnSlW3suvD8cIvitmb032ysPd+0PIfj0mSinfTZAXZCDPguHSzYAbwEPuHu//ZXF92Q6dUFu3pNXA98A4v3MH9H+KqaASJWiyZ8K0mkz0tJZ5zqg2t0PBh6k5xNCruWiv9LxJ4Lvl5kH/BD4dTZXbmYVwJ3AV9x9e/LsFItkpc8GqSsnfebune4+H5gFLDKzg5Ka5KS/0qgr6+9JMzsWeMvd6wZqluK5YfdXMQVEPZCY8rOALf21MbNSYBKZH8oYtC533+rubeHD64GFGa4pXen0ada5+/auIQJ3vxcoM7Np2Vi3mZURbIRXuftdKZrkpM8GqyuXfRausxF4BFiaNCsX78lB68rRe/JQYJmZ/Y1gKPooM7slqc2I9lcxBcRTwFwzm2NmYwgO4KxNarMWODmc/iTwkIdHe3JZV9IY9TKCMeR8sBb4fHhmzmKgyd1fz3VRZrZn17irmS0i+H++NQvrNeBnwEZ3/0E/zbLeZ+nUlYs+M7MqM6sMp8cCHwJeTGqW9fdkOnXl4j3p7he6+yx3rybYTjzk7iclNRvR/iqaO8q5e8zMzgbuJzhz6EZ3f8HMvgPUuvtagjfRzWb2MkHqrsyTur5sZsuAWFjXKZmuC8DMbiU4u2WamdUDFxMcsMPdrwXuJTgr52VgJ/CFPKnrk8AZZhYDdgErsxD0EHzC+xzwXDh+DXARsHdCbbnos3TqykWf7QXcZGYRgkC6zd3vyfV7Ms26cvKeTCWT/aWv2hARkZSKaYhJRESGQAEhIiIpKSBERCQlBYSIiKSkgBARkZQUECLDZGaVZnZmOD3DzO7IdU0iI0mnuYoMU/i9Rve4e/LXMIgUhKK5UE4kA64E9gsvPvsrcIC7H2RmpwCfILjw8SDg+wRf5f45oA34qLtvM7P9gB8DVQQXzX3R3ZOvJBbJGQ0xiQzfBcCm8Evdvp407yDgMwRf534FsNPdDwH+AHw+bHMdcI67LwS+BvxXVqoWSZP2IEQy4+Hw3gvNZtZE8O2fAM8BB4ffrPrPwO0JX9dfnv0yRfqngBDJjLaE6XjC4zjB+64EaAz3PkTykoaYRIavmeAWnkMW3o/hFTNbDt33qp43ksWJ7C4FhMgwuftW4H/M7Hngu8N4ic8C/2JmzwAvkCe3mhXpotNcRUQkJe1BiIhISgoIERFJSQEhIiIpKSBERCQlBYSIiKSkgBARkZQUECIiktL/B+wS8yrShjLAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "run_train(all_data, all_labels, backward_prop)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
